import time

import scrapy

from ..items import BoosproItem


class BoosSpider(scrapy.Spider):
    name = "boos"
    # allowed_domains = ["read.douban.com"]
    start_urls = ["https://movie.douban.com/review/best/"]

    url = 'https://movie.douban.com/review/best/?start=%d'

    num = None

    page_Num = 20

    def get_str(self, value):
        value = str(value).replace(' ', '')
        value = value.replace('\r', '')
        value = value.replace('\n', '')
        return value

    def pase_detail(self, response):
        self.num = response.url.split('/')[-2]
        item = response.meta['item']
        # item['job_desc']=
        # // *[ @ id = "link-report-16120527"] / div[1]
        # // *[ @ id = "link-report-16123223"] / div
        # // *[ @ id = "link-report-16123223"] / div
        # // *[ @ id = "link-report-16122030"] / div[1]
        a = '//*[@id="link-report-' + self.num + '"]//text()'
        print(f"num={self.num}")
        p_lists = response.xpath(a).extract()
        p_desc = ''.join(p_lists)
        item['job_desc'] = self.get_str(p_desc)
        yield item

    def parse(self, response):
        div_List = response.xpath('//*[@id="content"]/div/div[1]/div[1]/div')
        for div in div_List:
            item = BoosproItem()

            title = div.xpath('./div/a/img/@title')[0].get()
            item['job_name'] = title
            detail_url = div.xpath('./div/div/h2/a/@href')[0].get()
            # 对详情页发起请求
            print(title, detail_url)
            self.num = str(detail_url).split('/')[-2]
            print("parse_num=", self.num)

            # 请求传参:meta={} 可以将meta字典传递给请求对应的回调函数
            time.sleep(0.5)
            yield scrapy.Request(url=detail_url, callback=self.pase_detail, meta={'item': item})
        print(f"这是第{self.page_Num}的页面")
        if self.page_Num <= 80:
            new_url = format(self.url % self.page_Num)
            self.page_Num += 20
            yield scrapy.Request(url=new_url, callback=self.parse)


import os

if __name__ == "__main__":
    url = "https://movie.douban.com/review/16122239/"
    split_ = url.split('/')[-2]
    print(os.listdir('/第八章_Scrapy框架\\boospro\\boospro\\spiders'))
